{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "85e0bd4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from bayes_opt import BayesianOptimization\n",
    "import numpy as np\n",
    "\n",
    "x,y = make_classification(n_samples=1000, n_features=10, n_classes=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "54f28b5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9729987820512822\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier()\n",
    "print(np.mean(cross_val_score(rf, x, y, cv=20, scoring='roc_auc')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1be22fa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def rf_cv(n_estimators, min_samples_split, max_features, max_depth):\n",
    "    val = cross_val_score(\n",
    "        RandomForestClassifier(n_estimators=int(n_estimators),\n",
    "            min_samples_split=int(min_samples_split),\n",
    "            max_features=min(max_features, 0.999), \n",
    "            max_depth=int(max_depth),\n",
    "            random_state=2),\n",
    "    x, y, scoring='roc_auc',cv=5).mean()\n",
    "    return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b50a5d1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_bo = BayesianOptimization(\n",
    "        rf_cv,\n",
    "        {'n_estimators': (10, 250),\n",
    "        'min_samples_split': (2, 25),\n",
    "        'max_features':(0.1, 0.999),\n",
    "        'max_depth':(5,15)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "497eed4b",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   | max_depth | max_fe... | min_sa... | n_esti... |\n",
      "-------------------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m 0.9718  \u001b[0m | \u001b[0m 9.208   \u001b[0m | \u001b[0m 0.3476  \u001b[0m | \u001b[0m 23.25   \u001b[0m | \u001b[0m 174.4   \u001b[0m |\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m 0.9722  \u001b[0m | \u001b[95m 5.729   \u001b[0m | \u001b[95m 0.436   \u001b[0m | \u001b[95m 6.348   \u001b[0m | \u001b[95m 222.6   \u001b[0m |\n",
      "| \u001b[95m 3       \u001b[0m | \u001b[95m 0.9723  \u001b[0m | \u001b[95m 5.069   \u001b[0m | \u001b[95m 0.5003  \u001b[0m | \u001b[95m 18.08   \u001b[0m | \u001b[95m 140.3   \u001b[0m |\n",
      "| \u001b[95m 4       \u001b[0m | \u001b[95m 0.9736  \u001b[0m | \u001b[95m 9.81    \u001b[0m | \u001b[95m 0.906   \u001b[0m | \u001b[95m 16.96   \u001b[0m | \u001b[95m 161.0   \u001b[0m |\n",
      "| \u001b[95m 5       \u001b[0m | \u001b[95m 0.9755  \u001b[0m | \u001b[95m 9.375   \u001b[0m | \u001b[95m 0.7546  \u001b[0m | \u001b[95m 10.64   \u001b[0m | \u001b[95m 124.0   \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m 0.9739  \u001b[0m | \u001b[0m 9.878   \u001b[0m | \u001b[0m 0.8786  \u001b[0m | \u001b[0m 23.53   \u001b[0m | \u001b[0m 244.8   \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m 0.9723  \u001b[0m | \u001b[0m 12.58   \u001b[0m | \u001b[0m 0.3785  \u001b[0m | \u001b[0m 20.79   \u001b[0m | \u001b[0m 208.8   \u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m 0.975   \u001b[0m | \u001b[0m 9.371   \u001b[0m | \u001b[0m 0.4578  \u001b[0m | \u001b[0m 16.98   \u001b[0m | \u001b[0m 160.7   \u001b[0m |\n",
      "| \u001b[95m 9       \u001b[0m | \u001b[95m 0.9757  \u001b[0m | \u001b[95m 9.074   \u001b[0m | \u001b[95m 0.6156  \u001b[0m | \u001b[95m 10.77   \u001b[0m | \u001b[95m 123.8   \u001b[0m |\n",
      "| \u001b[95m 10      \u001b[0m | \u001b[95m 0.9763  \u001b[0m | \u001b[95m 9.627   \u001b[0m | \u001b[95m 0.6835  \u001b[0m | \u001b[95m 11.87   \u001b[0m | \u001b[95m 124.2   \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m 0.9583  \u001b[0m | \u001b[0m 9.526   \u001b[0m | \u001b[0m 0.1093  \u001b[0m | \u001b[0m 12.64   \u001b[0m | \u001b[0m 122.6   \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m 0.975   \u001b[0m | \u001b[0m 9.228   \u001b[0m | \u001b[0m 0.4668  \u001b[0m | \u001b[0m 12.49   \u001b[0m | \u001b[0m 125.6   \u001b[0m |\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m 0.9753  \u001b[0m | \u001b[0m 7.782   \u001b[0m | \u001b[0m 0.8311  \u001b[0m | \u001b[0m 10.87   \u001b[0m | \u001b[0m 124.7   \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m 0.9755  \u001b[0m | \u001b[0m 10.73   \u001b[0m | \u001b[0m 0.5492  \u001b[0m | \u001b[0m 11.33   \u001b[0m | \u001b[0m 124.9   \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m 0.9755  \u001b[0m | \u001b[0m 9.718   \u001b[0m | \u001b[0m 0.419   \u001b[0m | \u001b[0m 10.65   \u001b[0m | \u001b[0m 125.7   \u001b[0m |\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m 0.9753  \u001b[0m | \u001b[0m 7.655   \u001b[0m | \u001b[0m 0.8503  \u001b[0m | \u001b[0m 10.77   \u001b[0m | \u001b[0m 126.6   \u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m 0.975   \u001b[0m | \u001b[0m 8.734   \u001b[0m | \u001b[0m 0.4486  \u001b[0m | \u001b[0m 8.936   \u001b[0m | \u001b[0m 127.0   \u001b[0m |\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m 0.9696  \u001b[0m | \u001b[0m 10.47   \u001b[0m | \u001b[0m 0.2235  \u001b[0m | \u001b[0m 11.48   \u001b[0m | \u001b[0m 127.1   \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m 0.9712  \u001b[0m | \u001b[0m 7.735   \u001b[0m | \u001b[0m 0.3608  \u001b[0m | \u001b[0m 9.13    \u001b[0m | \u001b[0m 124.6   \u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m 0.9735  \u001b[0m | \u001b[0m 10.76   \u001b[0m | \u001b[0m 0.9922  \u001b[0m | \u001b[0m 9.821   \u001b[0m | \u001b[0m 125.0   \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m 0.9749  \u001b[0m | \u001b[0m 7.462   \u001b[0m | \u001b[0m 0.4528  \u001b[0m | \u001b[0m 9.486   \u001b[0m | \u001b[0m 128.2   \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m 0.975   \u001b[0m | \u001b[0m 8.883   \u001b[0m | \u001b[0m 0.8329  \u001b[0m | \u001b[0m 11.71   \u001b[0m | \u001b[0m 124.9   \u001b[0m |\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m 0.9656  \u001b[0m | \u001b[0m 6.517   \u001b[0m | \u001b[0m 0.2451  \u001b[0m | \u001b[0m 12.02   \u001b[0m | \u001b[0m 125.7   \u001b[0m |\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m 0.9752  \u001b[0m | \u001b[0m 6.988   \u001b[0m | \u001b[0m 0.7207  \u001b[0m | \u001b[0m 7.786   \u001b[0m | \u001b[0m 127.7   \u001b[0m |\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m 0.9724  \u001b[0m | \u001b[0m 9.317   \u001b[0m | \u001b[0m 0.3438  \u001b[0m | \u001b[0m 16.4    \u001b[0m | \u001b[0m 159.1   \u001b[0m |\n",
      "| \u001b[0m 26      \u001b[0m | \u001b[0m 0.9737  \u001b[0m | \u001b[0m 10.34   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 12.5    \u001b[0m | \u001b[0m 124.9   \u001b[0m |\n",
      "| \u001b[0m 27      \u001b[0m | \u001b[0m 0.9739  \u001b[0m | \u001b[0m 8.098   \u001b[0m | \u001b[0m 0.9793  \u001b[0m | \u001b[0m 7.719   \u001b[0m | \u001b[0m 129.0   \u001b[0m |\n",
      "| \u001b[0m 28      \u001b[0m | \u001b[0m 0.9685  \u001b[0m | \u001b[0m 8.151   \u001b[0m | \u001b[0m 0.2459  \u001b[0m | \u001b[0m 18.32   \u001b[0m | \u001b[0m 161.1   \u001b[0m |\n",
      "| \u001b[0m 29      \u001b[0m | \u001b[0m 0.9594  \u001b[0m | \u001b[0m 8.326   \u001b[0m | \u001b[0m 0.1191  \u001b[0m | \u001b[0m 9.955   \u001b[0m | \u001b[0m 126.1   \u001b[0m |\n",
      "| \u001b[0m 30      \u001b[0m | \u001b[0m 0.9741  \u001b[0m | \u001b[0m 10.41   \u001b[0m | \u001b[0m 0.9571  \u001b[0m | \u001b[0m 11.17   \u001b[0m | \u001b[0m 125.5   \u001b[0m |\n",
      "=========================================================================\n"
     ]
    }
   ],
   "source": [
    "rf_bo.maximize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "17d31d70",
   "metadata": {},
   "outputs": [],
   "source": [
    "index=[]\n",
    "for i in rf_bo.res:\n",
    "    index.append(i['target'])\n",
    "max_index = index.index(max(index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f22d688c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9762992879287928"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_bo.max['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "17626b36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 9.626963586558684,\n",
       " 'max_features': 0.6835293647496075,\n",
       " 'min_samples_split': 11.871331399595716,\n",
       " 'n_estimators': 124.1603744898579}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_bo.max['params']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "34ef7d3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_bo.probe({'n_estimators': [10,100,200],\n",
    "        'min_samples_split': [2, 10,20],\n",
    "        'max_features':[0.1, 0.5,0.9],\n",
    "        'max_depth':[5,10,15]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "439107b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9762992879287928"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "{'max_depth': 9.626963586558684,\n",
       " 'max_features': 0.6835293647496075,\n",
       " 'min_samples_split': 11.871331399595716,\n",
       " 'n_estimators': 124.1603744898579}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity='all'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9734aaaf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9762992879287928"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "{'max_depth': 9.626963586558684,\n",
       " 'max_features': 0.6835293647496075,\n",
       " 'min_samples_split': 11.871331399595716,\n",
       " 'n_estimators': 124.1603744898579}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_bo.max['target']\n",
    "rf_bo.max['params']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb8803b0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
